/*
 * Licensed to the Apache Software Foundation (ASF) under one or more
 * contributor license agreements.  See the NOTICE file distributed with
 * this work for additional information regarding copyright ownership.
 * The ASF licenses this file to You under the Apache License, Version 2.0
 * (the "License"); you may not use this file except in compliance with
 * the License.  You may obtain a copy of the License at
 *
 *    http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package org.apache.kyuubi.config

import java.nio.charset.StandardCharsets
import java.nio.file.{Files, Path, Paths, StandardOpenOption}

import scala.collection.JavaConverters._

import org.apache.kyuubi.KyuubiFunSuite
import org.apache.kyuubi.ha.HighAvailabilityConf

class AllKyuubiConfiguration extends KyuubiFunSuite {

  private val markdown = Paths.get("..", "docs", "deployment", "settings.md")
    .toAbsolutePath

  private val writer = Files.newBufferedWriter(
    markdown, StandardCharsets.UTF_8,
    StandardOpenOption.TRUNCATE_EXISTING,
    StandardOpenOption.CREATE)

  def writeWithNewLine(content: String): Unit = {
    writer.write(content)
    writer.newLine()
    writer.flush()
  }

  def writeWith2Line(content: String): Unit = {
    writeWithNewLine(content)
    writer.newLine()
  }

  def rewriteToConf(path: Path): Unit = {
    val env =
      Files.newBufferedReader(path, StandardCharsets.UTF_8)

    try {
      writeWithNewLine("```bash")
      var line = env.readLine()
      while(line != null) {
        writeWithNewLine(line)
        line = env.readLine()
      }
      writeWithNewLine("```")
    } finally {
      env.close()
    }
  }

  override def afterAll(): Unit = {
    writer.close()
    super.afterAll()
  }

  test("Check all kyuubi configs") {

    KyuubiConf
    HighAvailabilityConf

    writeWith2Line("<!-- DO NOT MODIFY THIS FILE DIRECTORY, IT IS AUTO GENERATED BY" +
      " [org.apache.kyuubi.config.AllKyuubiConfiguration] -->")

    writeWith2Line(
      """
        |<div align=center>
        |
        |![](../imgs/kyuubi_logo_simple.png)
        |
        |</div>
        |""".stripMargin)

    writeWith2Line("# Introduction to the Kyuubi Configurations System")

    writeWith2Line(
      "Kyuubi provides several ways to configure the system and corresponding engines.")

    writeWith2Line("## Environments")
    writeWith2Line("You can configure the environment variables in" +
      " `$KYUUBI_HOME/conf/kyuubi-env.sh`, e.g, `JAVA_HOME`, then this java runtime will be used" +
      " both for Kyuubi server instance and the applications it launches. You can also change" +
      " the variable in the subprocess's env configuration file, e.g." +
      "`$SPARK_HOME/conf/spark-env.sh` to use more specific ENV for SQL engine applications.")

    rewriteToConf(Paths.get("..", "conf", "kyuubi-env.sh.template"))

    writeWith2Line("## Kyuubi Configurations")
    writeWith2Line("You can configure the Kyuubi properties in" +
      " `$KYUUBI_HOME/conf/kyuubi-defaults.conf`. For example:")

    rewriteToConf(Paths.get("..", "conf", "kyuubi-defaults.conf.template"))

    KyuubiConf.kyuubiConfEntries.values().asScala
      .toSeq
      .groupBy(_.key.split("\\.")(1))
      .toSeq.sortBy(_._1).foreach { case (category, entries) =>

        writeWith2Line(s"### ${category.capitalize}")

        writeWithNewLine("Key | Default | Meaning | Since")
        writeWithNewLine("--- | --- | --- | ---")

        entries.sortBy(_.key).foreach { c =>
          val key = {
            val sb = new StringBuilder()
            var curLen = 0
            c.key.split("\\.").foreach { str =>
              if (curLen + str.length > 21) {
                sb.append("<br>\\." + str)
                curLen = str.length + 1
              } else {
                sb.append("\\." + str)
                curLen += (str.length + 1)
              }
            }
            sb.toString().stripPrefix("\\.")
          }
          val dft = c.defaultValStr.replace("<", "&lt;").replace(">", "&gt;")
          val seq = Seq(
            key,
            s"<div style='width: 80pt;word-wrap: break-word;white-space: normal'>$dft</div>",
            s"<div style='width: 200pt;word-wrap: break-word;white-space: normal'>${c.doc}</div>",
            s"<div style='width: 20pt'>${c.version}</div>")
          writeWithNewLine(seq.mkString("|"))
        }
        writer.newLine()
    }

    writeWith2Line("## Spark Configurations")
    writeWith2Line("### Via spark-defaults.conf")
    writeWith2Line("Setting them in `$SPARK_HOME/conf/spark-defaults.conf`" +
      " supplies with default values for SQL engine application. Available properties can be" +
      " found at Spark official online documentation for" +
      " [Spark Configurations](http://spark.apache.org/docs/latest/configuration.html)")

    writeWith2Line("### Via kyuubi-defaults.conf")
    writeWith2Line("Setting them in `$KYUUBI_HOME/conf/kyuubi-defaults.conf`" +
      " supplies with default values for SQL engine application too. These properties will" +
      " override all settings in `$SPARK_HOME/conf/spark-defaults.conf`")

    writeWith2Line("### Via JDBC Connection URL")
    writeWith2Line("Setting them in the JDBC Connection URL" +
      " supplies session-specific for each SQL engine. For example: " +
      "```" +
      "jdbc:hive2://localhost:10009/default;#" +
      "spark.sql.shuffle.partitions=2;spark.executor.memory=5g" +
      "```")
    writeWithNewLine("- **Runtime SQL Configuration**")
    writeWithNewLine("  - For [Runtime SQL Configurations](" +
      "http://spark.apache.org/docs/latest/configuration.html#runtime-sql-configuration), they" +
      " will take affect every time")
    writeWithNewLine("- **Static SQL and Spark Core Configuration**")
    writeWithNewLine("  - For [Static SQL Configurations](" +
      "http://spark.apache.org/docs/latest/configuration.html#static-sql-configuration) and" +
      " other spark core configs, e.g. `spark.executor.memory`, they will take affect if there" +
      " is no existing SQL engine application. Otherwise, they will just be ignored")
    writeWith2Line("### Via SET Syntax")
    writeWithNewLine("Please refer to the Spark official online documentation for" +
      " [SET Command](http://spark.apache.org/docs/latest/sql-ref-syntax-aux-conf-mgmt-set.html)")


    writeWith2Line("## Logging")
    writeWith2Line("Kyuubi uses [log4j](https://logging.apache.org/log4j/2.x/) for logging." +
      " You can configure it using `$KYUUBI_HOME/conf/log4j.properties`.")

    rewriteToConf(Paths.get("..", "conf", "log4j.properties.template"))

    writeWith2Line("## Other Configurations")

    writeWith2Line("### Hadoop Configurations")
    writeWith2Line("Specifying `HADOOP_CONF_DIR` to the directory contains hadoop configuration" +
      " files or treating them as Spark properties with a `spark.hadoop.` prefix." +
      " Please refer to the Spark official online documentation for" +
      " [Inheriting Hadoop Cluster Configuration](http://spark.apache.org/docs/latest/" +
      "configuration.html#inheriting-hadoop-cluster-configuration)." +
      " Also, please refer to the [Apache Hadoop](http://hadoop.apache.org)'s" +
      " online documentation for an overview on how to configure Hadoop.")
    writeWith2Line("### Hive Configurations")
    writeWith2Line("These configurations are used for SQL engine application to talk to" +
      " Hive MetaStore and could be configured in a `hive-site.xml`." +
      " Placed it in `$SPARK_HOME/conf` directory, or treating them as Spark properties with" +
      " a `spark.hadoop.` prefix.")

    writeWith2Line("## User Defaults")
    writeWith2Line("In Kyuubi, we can configure user default settings to meet separate needs." +
      " These user defaults override system defaults, but will be overridden by those from" +
      " [JDBC Connection URL](#via-jdbc-connection-url) or [Set Command](#via-set-syntax)" +
      " if could be. They will take effect when creating the SQL engine application ONLY.")
    writeWith2Line("User default settings are in the form of `___{username}___.{config key}`." +
      " There are three continuous underscores(`_`) at both sides of the `username` and" +
      " a dot(`.`) that separates the config key and the prefix. For example:")
    writeWithNewLine("```bash")
    writeWithNewLine("# For system defaults")
    writeWithNewLine("spark.master=local")
    writeWithNewLine("spark.sql.adaptive.enabled=true")
    writeWithNewLine("# For a user named kent")
    writeWithNewLine("___kent___.spark.master=yarn")
    writeWithNewLine("___kent___.spark.sql.adaptive.enabled=false")
    writeWithNewLine("# For a user named bob")
    writeWithNewLine("___bob___.spark.master=spark://master:7077")
    writeWithNewLine("___bob___.spark.executor.memory=8g")
    writeWith2Line( "```")
    writeWith2Line("In the above case, if there are related configurations from" +
      " [JDBC Connection URL](#via-jdbc-connection-url), `kent` will run his SQL engine" +
      " application on YARN and prefer the Spark AQE to be off, while `bob` will activate" +
      " his SQL engine application on a Spark standalone cluster with 8g heap memory for each" +
      " executor and obey the Spark AQE behavior of Kyuubi system default. On the other hand," +
      " for those users who do not have custom configurations will use system defaults.")
  }
}
